
/*** This do-file converts MARTS flat files to Stata dta.
***/

*-------------------------------------------------------------------------------
* Set up
*-------------------------------------------------------------------------------

* Set $root 
project figstabs, root
if (r(buildrunning)==0) include "${root}/code/config_interactive.do"

* Create required subfolders
cap mkdir "${root}/data/derived/MARTS"

project, uses("${root}/data/dvc/MARTS/source.yaml") reference
project, uses("${root}/data/dvc/MARTS/MARTS-mf.zip") reference

*-------------------------------------------------------------------------------
* Time Periods
*-------------------------------------------------------------------------------

project, uses("${root}/data/dvc/MARTS/MARTS_time_periods.csv")
import delimited using "${root}/data/dvc/MARTS/MARTS_time_periods.csv", varnames(1) clear

gen long month = monthly(per_name, "MY")
format %tm month
drop if mi(month)
drop per_name

destring per_idx, replace
gisid per_idx
tempfile per_idx
save `per_idx'

*-------------------------------------------------------------------------------
* Categories
*-------------------------------------------------------------------------------

project, uses("${root}/data/dvc/MARTS/MARTS_categories.csv")
import delimited using "${root}/data/dvc/MARTS/MARTS_categories.csv", clear

gisid cat_idx
tempfile cat_idx
save `cat_idx'

*-------------------------------------------------------------------------------
* Data Types
*-------------------------------------------------------------------------------

project, uses("${root}/data/dvc/MARTS/MARTS_data_types.csv")
import delimited using "${root}/data/dvc/MARTS/MARTS_data_types.csv", clear

cap rename dat_desc dt_desc
gisid dt_idx
tempfile dt_idx
save `dt_idx'

*-------------------------------------------------------------------------------
* Data
*-------------------------------------------------------------------------------

* Load data
project, uses("${root}/data/dvc/MARTS/MARTS_data.csv")
import delimited using "${root}/data/dvc/MARTS/MARTS_data.csv", asdouble clear

assert geo_idx == 1 // all estimates are national
drop geo_idx

label define seasonal_adj 1 "seasonally adjusted" 0 "NOT seasonally adjusted"
label values is_adj seasonal_adj

* Drop error estimates (SEs and CVs)
assert dt_idx == 0 if et_idx != 0
assert et_idx == 0 if dt_idx != 0

drop if et_idx != 0
drop et_idx

* Merge in time periods
merge m:1 per_idx using `per_idx', keepusing(month) assert(2 3) // file structure changed so that sometimes there are extra observations in the time periods data

* Check any extra time period observations are from the end of the sample
sum month if _merge == 2
if r(N) > 0 local min_using = `r(min)'
else local min_using = .  // given line 85, we purposely let this be missing so the assertion is correct whether or not we have extra time periods

sum month if _merge == 3
local max_merged = `r(max)'
assert `min_using' > `max_merged'

drop if _merge == 2 // drop extra time period observations
drop _merge
drop per_idx

* Merge in categories
merge m:1 cat_idx using `cat_idx', keepusing(cat_code cat_desc) assert(3) nogen
drop cat_idx

* Merge in data types
merge m:1 dt_idx using `dt_idx', keepusing(dt_desc dt_unit) assert(3) nogen
drop dt_idx

* Output monthly sales
keep if dt_desc == "Sales - Monthly"
assert dt_unit == "MLN$"
drop dt_desc dt_unit

rename val sales
label var sales "Monthly sales, in millions $"

gisid is_adj cat_code month
sort is_adj cat_code month
order sales is_adj cat_code month

compress
save "${root}/data/derived/MARTS/MARTS monthly sales by industry.dta", replace
project, creates("${root}/data/derived/MARTS/MARTS monthly sales by industry.dta")
